{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "lesbian-springer",
   "metadata": {},
   "source": [
    "## Seldon Model Zoo\n",
    "\n",
    "Examples of various model artefact types from various frameworks running under Seldon Core 2.\n",
    "\n",
    " * SKlearn\n",
    " * Tensorflow\n",
    " * XGBoost\n",
    " * ONNX\n",
    " * Lightgbm\n",
    " * MLFlow\n",
    " * PyTorch\n",
    "\n",
    "Python requirements in `model-zoo-requirements.txt`"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5b1925df",
   "metadata": {},
   "source": [
    "### SKLearn Iris Classification Model\n",
    "\n",
    "The training code for this model can be found at `./scripts/models/iris`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "8b4be911",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "apiVersion: mlops.seldon.io/v1alpha1\r\n",
      "kind: Model\r\n",
      "metadata:\r\n",
      "  name: iris\r\n",
      "spec:\r\n",
      "  storageUri: \"gs://seldon-models/scv2/samples/mlserver_1.3.5/iris-sklearn\"\r\n",
      "  requirements:\r\n",
      "  - sklearn\r\n",
      "  memory: 100Ki\r\n"
     ]
    }
   ],
   "source": [
    "!cat ./models/sklearn-iris-gs.yaml"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "checked-cream",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{}\r\n"
     ]
    }
   ],
   "source": [
    "!seldon model load -f ./models/sklearn-iris-gs.yaml"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "interracial-secret",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{}\r\n"
     ]
    }
   ],
   "source": [
    "!seldon model status iris -w ModelAvailable | jq -M ."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "upper-cholesterol",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\r\n",
      "\t\"model_name\": \"iris_1\",\r\n",
      "\t\"model_version\": \"1\",\r\n",
      "\t\"id\": \"09263298-ca66-49c5-acb9-0ca75b06f825\",\r\n",
      "\t\"parameters\": {},\r\n",
      "\t\"outputs\": [\r\n",
      "\t\t{\r\n",
      "\t\t\t\"name\": \"predict\",\r\n",
      "\t\t\t\"shape\": [\r\n",
      "\t\t\t\t1,\r\n",
      "\t\t\t\t1\r\n",
      "\t\t\t],\r\n",
      "\t\t\t\"datatype\": \"INT64\",\r\n",
      "\t\t\t\"data\": [\r\n",
      "\t\t\t\t2\r\n",
      "\t\t\t]\r\n",
      "\t\t}\r\n",
      "\t]\r\n",
      "}\r\n"
     ]
    }
   ],
   "source": [
    "!seldon model infer iris \\\n",
    "  '{\"inputs\": [{\"name\": \"predict\", \"shape\": [1, 4], \"datatype\": \"FP32\", \"data\": [[1, 2, 3, 4]]}]}' "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "ceramic-illness",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{}\r\n"
     ]
    }
   ],
   "source": [
    "!seldon model unload iris"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "30431134",
   "metadata": {},
   "source": [
    "### Tensorflow CIFAR10 Image Classification Model\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "f0e7043f",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-03-09 19:43:43.637892: W tensorflow/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory\n",
      "2023-03-09 19:43:43.637906: I tensorflow/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine.\n"
     ]
    }
   ],
   "source": [
    "import requests\n",
    "import json\n",
    "from typing import Dict, List\n",
    "import numpy as np\n",
    "import os\n",
    "import tensorflow as tf\n",
    "from alibi_detect.utils.perturbation import apply_mask\n",
    "from alibi_detect.datasets import fetch_cifar10c\n",
    "import matplotlib.pyplot as plt\n",
    "tf.keras.backend.clear_session()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "727e6a62",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(50000, 32, 32, 3) (50000, 1) (10000, 32, 32, 3) (10000, 1)\n"
     ]
    }
   ],
   "source": [
    "train, test = tf.keras.datasets.cifar10.load_data()\n",
    "X_train, y_train = train\n",
    "X_test, y_test = test\n",
    "\n",
    "X_train = X_train.astype('float32') / 255\n",
    "X_test = X_test.astype('float32') / 255\n",
    "print(X_train.shape, y_train.shape, X_test.shape, y_test.shape)\n",
    "classes = (\n",
    "    \"plane\",\n",
    "    \"car\",\n",
    "    \"bird\",\n",
    "    \"cat\",\n",
    "    \"deer\",\n",
    "    \"dog\",\n",
    "    \"frog\",\n",
    "    \"horse\",\n",
    "    \"ship\",\n",
    "    \"truck\",\n",
    ")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "e2dc7fe8",
   "metadata": {},
   "outputs": [],
   "source": [
    "reqJson = json.loads('{\"inputs\":[{\"name\":\"input_1\",\"data\":[],\"datatype\":\"FP32\",\"shape\":[]}]}')\n",
    "url = \"http://0.0.0.0:9000/v2/models/model/infer\"\n",
    "\n",
    "def infer(resourceName: str, idx: int):\n",
    "    rows = X_train[idx:idx+1]\n",
    "    show(rows[0])\n",
    "    reqJson[\"inputs\"][0][\"data\"] = rows.flatten().tolist()\n",
    "    reqJson[\"inputs\"][0][\"shape\"] = [1, 32, 32, 3]\n",
    "    headers = {\"Content-Type\": \"application/json\", \"seldon-model\":resourceName}\n",
    "    response_raw = requests.post(url, json=reqJson, headers=headers)\n",
    "    probs = np.array(response_raw.json()[\"outputs\"][0][\"data\"])\n",
    "    print(classes[probs.argmax(axis=0)])\n",
    "    \n",
    "\n",
    "def show(X):\n",
    "    plt.imshow(X.reshape(32, 32, 3))\n",
    "    plt.axis(\"off\")\n",
    "    plt.show()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "34165288",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "apiVersion: mlops.seldon.io/v1alpha1\r\n",
      "kind: Model\r\n",
      "metadata:\r\n",
      "  name: cifar10\r\n",
      "spec:\r\n",
      "  storageUri: \"gs://seldon-models/scv2/samples/tensorflow/cifar10\"\r\n",
      "  requirements:\r\n",
      "  - tensorflow\r\n"
     ]
    }
   ],
   "source": [
    "!cat ./models/cifar10-no-config.yaml"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "726f2dd0",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{}\r\n"
     ]
    }
   ],
   "source": [
    "!seldon model load -f ./models/cifar10-no-config.yaml"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "2d9b631f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{}\r\n"
     ]
    }
   ],
   "source": [
    "!seldon model status cifar10 -w ModelAvailable | jq -M ."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "16ac398b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAGFCAYAAAASI+9IAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/av/WaAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAYW0lEQVR4nO3cya8lh1XH8VNVt+745qmH1+0eHNPIJnbkRCQxJMFxIoECiCBlmQ1/BvA3EKREgAQSC2AVoYhECVJghZQYlMGyY7ud2HG759f95vfuXBOLRmfr80NtEaPvZ336dL2quvd3a1G/pGmaxgAAMLP0//oAAAC/OggFAIAjFAAAjlAAADhCAQDgCAUAgCMUAACOUAAAuFZ08JvffVlaXNd1eLbX6Ui7291u/DgybXfZxHOyZZm0O6vis3n89D0ivoPYtOJ/Z5GIu4XZtBLfnWzy8GhZaLurVLhAZmaJNq5Q3imV3z8VjruuxXMoLFffmlX/TuU7qKrEay9Q/85SuvbaF8Wf/OEz7zvDkwIAwBEKAABHKAAAHKEAAHCEAgDAEQoAAEcoAAAcoQAAcIQCAMARCgAARygAAFy4+6gWe15anXhHzbzWekdGx6fh2XygHXiW9+LDjba7FnphSrFvqJoW0vz0eBKebXe1/qjK4n0sw8lQ2p0m8WNZGCxLuxvhuM3MaqEvJ0m0e0W5+monkHJrqd1Hyj2uVjYpXUaP9sf/A7X7SLmetdh+VAvHrZ6TCJ4UAACOUAAAOEIBAOAIBQCAIxQAAI5QAAA4QgEA4AgFAIAjFAAAjlAAALhwzcXJSKsjKIp47cLe7r60+87dh+HZrDuQdi8sroZnO6lW/6C0YsxLrbaiLkppfnwav569XPs7LY2/en86j1eWmJnN5/GTePXKU9Lujzx5SZrvdbvhWbWOQJoXK2ga4R/UYt2K0uig1nOo8x8kpeYiFS9QLdatPG48KQAAHKEAAHCEAgDAEQoAAEcoAAAcoQAAcIQCAMARCgAARygAAByhAABwhAIAwIW7j374ny9Li4dCV1JqubR7Mot3oEwrrVcpb8fns1rL1EqoQJk2WpdRJXbUDNrx3p5eEr5NzMys28nCs1U6l3aPRvFOqB+/9oq0++HePWn+6pUr4dmNjQ1pd6/fD882tXbtq6oKz9aN1sOTKJ+JX6EuI1UjdFM1Qk+SmdbxpHZqRfCkAABwhAIAwBEKAABHKAAAHKEAAHCEAgDAEQoAAEcoAAAcoQAAcIQCAMCF+wuOhhNpcdPEX+1OTHvdvdWO12L0xYqGLI3Pt60t7Z5avF6gFPP6dDyS5iej+HwniddWmJktNJ3wbKZdHss7vfDsdDiVdv/y9l1p/ub9nfDsytKytPvihQvh2c2NdWn3yupqeLaVatc+E2oxlDqH/41KWF/bB1dF0YhVIbVUc/H4zyFPCgAARygAAByhAABwhAIAwBEKAABHKAAAHKEAAHCEAgDAEQoAAEcoAAAcoQAAcOHmmclc6+/Ic6XURuwdqYr4rMVnzcySLN5PlIi1I/Mi3sVTiJ1Ai/0Faf70ZByePZlrvVezOn6vtNtaf9RiO37Ss0zbPSpn0nxWx39TzfaOpd1HR8Pw7GAh3gdlZnbu3Pnw7JNXrkq7F9rx3quOeO2LQvssF8JXVmNax1P9AXY8KeNKv1MUTwoAAEcoAAAcoQAAcIQCAMARCgAARygAAByhAABwhAIAwBEKAABHKAAAXLzmYhavaDAzmxXxvEkSreai2+2GZ9W3wBvhUGqx50KZH43iNQdmZt2edg47efy1/qrQdk9n8VqMMtHqUxrhHLZTrbpA/4kUP5ZWSzsW5e88HWv3yvHb18Oze/t70u7F7nJ49sL2BWn36uqqNN/uKPUf2j1el2V4ttRucSuFG7Fq4rU8UTwpAAAcoQAAcIQCAMARCgAARygAAByhAABwhAIAwBEKAABHKAAAHKEAAHCEAgDAhbuP5o1W4JFU8fm61nbXqdZTIunEdzeZlql1Gu9LaYWvzCPFPN43ZGbWbsX7oxZ6bWn3eB7vySotfk7MzGZC3dSs1LqpOql20jOL9xk14u+voha6dUzrv0nT+LHsHDyUdt+b7Ydn37l5S9q9ubkhzZ8/fzE8u7CwKO3udoT+NbGDq2iE7qOK7iMAwAeIUAAAOEIBAOAIBQCAIxQAAI5QAAA4QgEA4AgFAIAjFAAAjlAAALjwe/2lWHOhqIRX+s3MpsPT8GxL7IuohAaNVjqXdjfC7jzXqjxa8Uv5iFItkmh1EQvtPDxbij9LamG+EOtTykq7nmkSP5im1I6lEqorqky7PkorRiOuThLh2hfaOTm5dyjN37z/Xni2047XVpiZ9fv98Gy3q+3utOO1MnkeP9+PPPu+EzwpAAAcoQAAcIQCAMARCgAARygAAByhAABwhAIAwBEKAABHKAAAHKEAAHCEAgDAhQtzZoXWC5Mk8e6eutYKVhqhkKWcTaTdk9k4PJsLHT9mZpnQldNpabubROuRSZosPFuLHUJNHS/XES+9jat4T9bctONO0/g5MTObC/d4rhRfmVmTxo+9SIUyI9P6jNJMOyeWTOO7xZ+k4q1itVCUNZ8Mpd0nI+Gci51aNosfi/I9+8hX33eCJwUAgCMUAACOUAAAOEIBAOAIBQCAIxQAAI5QAAA4QgEA4AgFAIAjFAAAjlAAALhw99F4Gu80MTNrKcUmdfgw/mc+3gszGT2QVrfb8YaVtTMXpN09oS4lFTp+zMyyXluab9IiPHt8uC/tngxPwrOXrlyTdp8Wg/Ds4eGxtLvT6UvzhdAHlpjWT1QrBUXarSLtrsTCobbF76s00w68LLSen0roPjKhl8zMrJmNwrP10W1p9/7dd4UDefy/63lSAAA4QgEA4AgFAIAjFAAAjlAAADhCAQDgCAUAgCMUAACOUAAAOEIBAODC/RJVKb5LL7wev9rpSauXBvE6gklfrNBI4tUF+XAire6W8Qze2tqSdk97XWl+XsbrCHpdrf4h68evZ39pSdq9MjgXnj27MZN210J9ipnZVKiLGIu7d3bj9SzF6EjanTfxa98qtXqbrI5/foriVNrdyrT7sLb4Z6JOxe+JSfzYT+69J62eHcav/XCo3eMRPCkAAByhAABwhAIAwBEKAABHKAAAHKEAAHCEAgDAEQoAAEcoAAAcoQAAcIQCAMDFCz/KeKeJmdlyfzE8uyL2E929fys8O2l3pN2zKt7xlOzclHZfWY/3GW1d3JZ2v3XvnjTf1El4tj/SOp6WB/HOmZ/dflXavXB2FJ/t5NLuG794U5qvBqvh2ZWnnpV2L5z/SHh2dPO6tDsbnoRnl5qhtHs8PIrPnj6UdrfzBWn+ZJqFZ3srm9Lu9V788zO0eNeUmZnFV1uSPv7f9TwpAAAcoQAAcIQCAMARCgAARygAAByhAABwhAIAwBEKAABHKAAAHKEAAHDhfom00l7VPrsQfyX9waH2unuxGH8PvLUYr9swM0uT+KvxZXEo7b70/DPh2UOrpd3z1b40nyXxapF0KV5bYWZ2dHIanj2dahUa9fgoPDubxitLzMyWxb/z9jBeATHa3Zd2X1pZCc+ev6ZVaBy9OQ3Pju5qVS6HD+LzJyPtnFSl9hv2eBL/nuitajUXixfj8+U4XitiZjadzMKzaRr/vgrvfOwbAQAfWoQCAMARCgAARygAAByhAABwhAIAwBEKAABHKAAAHKEAAHCEAgDAEQoAABcuwFlb0jqENhbi80cHD6Tda908PNvJ4/0nZmZlEe/L2XrymrT76rmL4dk3br0r7V7ptKX5spiHZ7fOrki7041479Wopf0uSRfjf+fh7o60+9LWBWl+3I6fw8NqJO0+ONwNz6bnnpB2X3j6U+HZu3feknZPJ+PwbJ5pn82maqT5rI73tc2OtP61XYv3e5Xj+DkxM0uz+GeiqqTVsf//8a8EAHxYEQoAAEcoAAAcoQAAcIQCAMARCgAARygAAByhAABwhAIAwBEKAAAXrrm4dHZNWvzHv/f58OzNdy9Lu0+nw/DsbBqvIjAzK2fxmovL57V6gaaOv6bfbJyVdh8LtRVmZqNx/Bxe2NiSdpdNHZ4djqbS7qbbCc8uNKvS7qzWOgPOLPfCs6OH8doKM7Ph3Xg1QjGLn28zs8GZeJ3H+Wc+I+2ui+Pw7MN7v5R2j4fxaolHBxO/nkuDTFrdskl4tgl/yz5SjOPH3ZhWFRLBkwIAwBEKAABHKAAAHKEAAHCEAgDAEQoAAEcoAAAcoQAAcIQCAMARCgAARygAAFy4lWMp0zpqPv18vBfoN5/Zlnafjmfh2aLRcq8o4/1E5Tjef2JmNpnGj/vKXDsn45nW2zMcxY89z7XylsOTk/Bs90pb2j2Zxc9hs7Ih7b67c1+af/vGrfDs06taf9St3YP4cK319lTdxfDswqXnpd2fefJyePbgttZ99POf/kSaf7jz8/DsIDmUdttsFB6dVtr1Sep4l1Ur13ZH8KQAAHCEAgDAEQoAAEcoAAAcoQAAcIQCAMARCgAARygAAByhAABwhAIAwBEKAAAXLrUZHmjdIHduvB6evbB9Rdq9fe5MeLbVj/e8mJnVSbzn52RvT9p9dBQ/h+tr69Lu0aSQ5seTeXz3MN7zYmZ2OlwOz1578qq0ezQSOmcmWjfVZq8jzeez+Dn/+CdfkHYfjOO739s5lnbP0254tpponWe2uhkePf+s9rnffPaL0nx5+CA8e3D9v6TdN17/UXh275e/kHan7fg9nrbiPUnhnY99IwDgQ4tQAAA4QgEA4AgFAIAjFAAAjlAAADhCAQDgCAUAgCMUAACOUAAAuHCnw0pvIC0+3d8Jz96vtVe1N84m4dnlLF5bYWY2WFyJDy9rFRpZEq8uWOxJq215QTuWJm2HZ8siXolhZnb9zbfCs5ub8VoEM7N+/4nw7Fis53ju8rY0/7lPPB+enZSNtHtcxmefulhJux/sx+s/7u0cSLt3btwOz96qtHMyFStreisXwrMrv/G70u6PXft0eHb7xmvS7td++L3w7O7ODWl3BE8KAABHKAAAHKEAAHCEAgDAEQoAAEcoAAAcoQAAcIQCAMARCgAARygAAByhAABw4WKgc2vL0uJkHu/5OXjwUNr96mvvhGdfef3n0u4z2xfDs5/53Gel3dub8XM4PRxLu7OWWJYkdB+1Wlp/1BPnV8OzvW4u7e60479jltp9abctxs+JmVlRxf/O00n882BmNqni/V7X335P2n042w3PPn9V66YabsXvlRv34/1oZmbXb8Y7tczMXn03/j1x2lmRdm8sxe+tp89onVqf+OwXw7OvvPxv0u4InhQAAI5QAAA4QgEA4AgFAIAjFAAAjlAAADhCAQDgCAUAgCMUAACOUAAAuPA76a+98iNpcbN/Mzy7vK69Sv+TN+Kvu78lVgD81osvhWf/8Z/+Qdr9By/9dnh2tdtIu7u9RWm+lcdf059MtcqNzfWt8GzdGUi7D2czaV6RZNpvpEL4TZXkXWn3OzfvhGe/9hdfk3bvPTwIz37yU/F71szs97/y1fDs1lntcz8oJ9L8+TJeFfLGUS3trtMyPPvwVvy70MzsqSfOhGevXnta2h3BkwIAwBEKAABHKAAAHKEAAHCEAgDAEQoAAEcoAAAcoQAAcIQCAMARCgAARygAAFy4+2j3SOu/eSvfDc9mD/el3bfu3w/Pfval35F2/+mf/1l49uvf+Ctp93e/8+3w7K9vr0u783YmzQ8Wl8KzVVVJu9eW18Kzm2vxnhczs1YrfMtau92WdqdJfLeZ2bCK99/MW9rvr7/+m78Pz7751s+k3Z08fl6+9e1vSrsvXPtoePajT/2atLvX0fqjlpr49Tm/IK22UrieoyrewWRm1szj/V6Xtp+QdkfwpAAAcIQCAMARCgAARygAAByhAABwhAIAwBEKAABHKAAAHKEAAHCEAgDAhd/r3778EWlxZafh2aKYSrvbg/g76ecubku7m6QJz148f0Ha/e//8s/h2dOdVWl3v9eR5ju9njCtvabfaeXh2YW+1i/Q7/XDs22hzsHMrNtWzolZ042f891J/PNgZvbG9TfDs1/4wkvS7uc+9lx49m//Ll63YWb28n/8a3j26tkVaXe7r1W57O3shGdfffsX0u58EL9XziytSLurSbxWptd+/L/reVIAADhCAQDgCAUAgCMUAACOUAAAOEIBAOAIBQCAIxQAAI5QAAA4QgEA4AgFAIALdx+VFu/jMDOr6niHULsT77MxMxssxWdPhmNp94OHu+HZvYNDafednf3wbFMW0u5uR+vtKYr49YxfyUc6efi2skEn3pNkZpa14v03vW5X2t3tavdhncU7oW7tPpB2WxPf/Udf/rK0+oUXXgjP3r59R9r9rW9/Jzz7yquXpN3VdC7NHz44Ds/O9+9Ku1vVYnh2XA6l3e8e3g7P9jtav1cETwoAAEcoAAAcoQAAcIQCAMARCgAARygAAByhAABwhAIAwBEKAABHKAAAXLiPYO8oXtFgZlaU0/hBpFo2NWW8ouGV116Xdn/0uY8Lu38m7S6EDJ63tNqKeRGvfzAzu39/Lzw7ncWvpZlZuxWvuci1w7Z4+YNZ3tYqNHKhnsPMrGrq8OxwOpF2r22cCc9urK9Lu09PTsKzZ8+dlXYfHMZrYr7//e9Ju6fDkTS/vx+vlxgl2ndQq9cJz2ZCZYmZ2eqZzfDs1hnt+kTwpAAAcIQCAMARCgAARygAAByhAABwhAIAwBEKAABHKAAAHKEAAHCEAgDAEQoAABcue6mSeM+LmVmStcOzw/FY2j0ZxjtNdna1zqa//Po3wrM337kp7R7O451N79yNd8iYmTV1I81XVfxYikq89tUsPJuJv0sSof0omcT/RjOzJinFY1GWa9enN4ifw/197R7vtOOfzZPjeE+SmdlsFj+H7713R9qdCJ1nZmaFcNs23b60W7ma7Tx+vs3MBp2F8Ox4pJ2TCJ4UAACOUAAAOEIBAOAIBQCAIxQAAI5QAAA4QgEA4AgFAIAjFAAAjlAAADhCAQDgwt1Ha+tr4uosPDkZjqTNs0G8GyRNtNw7OjwKz65vbkm7l9c2w7Ol2GVUN3Npvizi3TpVqXUCFUW8j6UuPrjOptlMOye12E9kTbxcJxV/fx2dxDuHfvDDH0i7X3zxxfDsG29el3YLl8fm4j2eCd8pZma18NlX+72qWREfnmt/5+2bt8OzWWdR2h3BkwIAwBEKAABHKAAAHKEAAHCEAgDAEQoAAEcoAAAcoQAAcIQCAMARCgAAlzRN7N3+L33lS9LiWnlrXHg13swsi7dzWKsVnzUzS5Q30kvtwGvhtf40017pL+dj7ViqeAVEJVYA1MLFV5slyiJeuTEcDaXds1m8+sPMrCiEcyjeK8qx9Hs9afflK1fCsz/+yU+l3Ucn0/BsYom0O/hV5SphvtEOxSxR/0FcmsY/+91+X9o9Ot57//9f2ggA+H+NUAAAOEIBAOAIBQCAIxQAAI5QAAA4QgEA4AgFAIAjFAAAjlAAADhCAQDgwsVASaJ18eR5PG+STOwRqeLzeZ5ru4V6lUbsP+kofUbi7rZW8WSJdcOzSt+QmVmlFF+JfTZKJ9T6xpq0uxD/zqaJ/516f1S8K2k00nqvdh48CM9evhzvSTIzOx0V4dnxZCLtlj6cZlYK91YlXEszs0a4x9UeszSNf3em6ePvYOJJAQDgCAUAgCMUAACOUAAAOEIBAOAIBQCAIxQAAI5QAAA4QgEA4AgFAIALlyM0jfaqdlPHX79OTHtVW2mAqJXKBRNrMVraOUmEA0/Fmgv1WDLhVfq81uoFiiJedVBV8ToHMzPlVmnE484SrRKlrOK1GGLTgeXC9ektrki7t59oh2dr8RxO5vHrqdaKqJ/lJIufw0asW1GOJRMvvvKZmM1m0u4InhQAAI5QAAA4QgEA4AgFAIAjFAAAjlAAADhCAQDgCAUAgCMUAACOUAAAOEIBAODC3UfzqdZRo/T8CBUlZqb1wqh9KVkrfEosEfuGGov3q9TCrJlZkmgnMRV6fvKe1gnUZPHuo4568SVaf5Taf1OW8e6eYj6XdtdN/L5VjsPMbDyP71a7qaZl/Nor3xFmZpaJ11M49kb8nmi34/1RLeE7RdXv9x/7Tp4UAACOUAAAOEIBAOAIBQCAIxQAAI5QAAA4QgEA4AgFAIAjFAAAjlAAALjw+9dNI76SLlQMVKX2Kr0l8flOpyOtLor4a/pVFZ81M8vb8boItZ6jZVoVRVXEqxFKrf1BqotQ6zzSNH5fqTUKiVCfYmaWd+I1J1ker0Uw045draJQ7q1CqK0wM0vr+H1Vi8ddivOZ8J1Vi1Uhyj2u1qcoUvGeDe187BsBAB9ahAIAwBEKAABHKAAAHKEAAHCEAgDAEQoAAEcoAAAcoQAAcIQCAMARCgAAlzQfZDEHAOBDhScFAIAjFAAAjlAAADhCAQDgCAUAgCMUAACOUAAAOEIBAOAIBQCA+28YKigFYDWmdwAAAABJRU5ErkJggg==",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "car\n"
     ]
    }
   ],
   "source": [
    "infer(\"cifar10\",4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "f7405940",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{}\r\n"
     ]
    }
   ],
   "source": [
    "!seldon model unload cifar10"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "04a726c0",
   "metadata": {},
   "source": [
    "### XGBoost Model\n",
    "\n",
    "The training code for this model can be found at `./scripts/models/income-xgb`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "c57d72d2",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "apiVersion: mlops.seldon.io/v1alpha1\r\n",
      "kind: Model\r\n",
      "metadata:\r\n",
      "  name: income-xgb\r\n",
      "spec:\r\n",
      "  storageUri: \"gs://seldon-models/scv2/samples/mlserver_1.3.5/income-xgb\"\r\n",
      "  requirements:\r\n",
      "  - xgboost\r\n"
     ]
    }
   ],
   "source": [
    "!cat ./models/income-xgb.yaml"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "28e5f8d9",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{}\r\n"
     ]
    }
   ],
   "source": [
    "!seldon model load -f ./models/income-xgb.yaml"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "8e20ab04",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{}\r\n"
     ]
    }
   ],
   "source": [
    "!seldon model status income-xgb -w ModelAvailable | jq -M ."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "a772f985",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\r\n",
      "\t\"model_name\": \"income-xgb_1\",\r\n",
      "\t\"model_version\": \"1\",\r\n",
      "\t\"id\": \"e30c3b44-fa14-4e5f-88f5-d6f4d287da20\",\r\n",
      "\t\"parameters\": {},\r\n",
      "\t\"outputs\": [\r\n",
      "\t\t{\r\n",
      "\t\t\t\"name\": \"predict\",\r\n",
      "\t\t\t\"shape\": [\r\n",
      "\t\t\t\t1,\r\n",
      "\t\t\t\t1\r\n",
      "\t\t\t],\r\n",
      "\t\t\t\"datatype\": \"FP32\",\r\n",
      "\t\t\t\"data\": [\r\n",
      "\t\t\t\t-1.8380107879638672\r\n",
      "\t\t\t]\r\n",
      "\t\t}\r\n",
      "\t]\r\n",
      "}\r\n"
     ]
    }
   ],
   "source": [
    "!seldon model infer income-xgb \\\n",
    "  '{ \"parameters\": {\"content_type\": \"pd\"}, \"inputs\": [{\"name\": \"Age\", \"shape\": [1, 1], \"datatype\": \"INT64\", \"data\": [47]},{\"name\": \"Workclass\", \"shape\": [1, 1], \"datatype\": \"INT64\", \"data\": [4]},{\"name\": \"Education\", \"shape\": [1, 1], \"datatype\": \"INT64\", \"data\": [1]},{\"name\": \"Marital Status\", \"shape\": [1, 1], \"datatype\": \"INT64\", \"data\": [1]},{\"name\": \"Occupation\", \"shape\": [1, 1], \"datatype\": \"INT64\", \"data\": [1]},{\"name\": \"Relationship\", \"shape\": [1, 1], \"datatype\": \"INT64\", \"data\": [3]},{\"name\": \"Race\", \"shape\": [1, 1], \"datatype\": \"INT64\", \"data\": [4]},{\"name\": \"Sex\", \"shape\": [1, 1], \"datatype\": \"INT64\", \"data\": [1]},{\"name\": \"Capital Gain\", \"shape\": [1, 1], \"datatype\": \"INT64\", \"data\": [0]},{\"name\": \"Capital Loss\", \"shape\": [1, 1], \"datatype\": \"INT64\", \"data\": [0]},{\"name\": \"Hours per week\", \"shape\": [1, 1], \"datatype\": \"INT64\", \"data\": [40]},{\"name\": \"Country\", \"shape\": [1, 1], \"datatype\": \"INT64\", \"data\": [9]}]}' "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "01f1d42b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{}\r\n"
     ]
    }
   ],
   "source": [
    "!seldon model unload income-xgb"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ca3f0a3c",
   "metadata": {},
   "source": [
    "## ONNX MNIST Model\n",
    "\n",
    "This model is a pretrained model as defined in `./scripts/models/Makefile` target `mnist-onnx`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "322251ec",
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import json\n",
    "import requests\n",
    "from torchvision.datasets import MNIST\n",
    "from torchvision.transforms import ToTensor\n",
    "from torchvision import transforms\n",
    "from torch.utils.data import DataLoader\n",
    "import numpy as np\n",
    "training_data = MNIST(\n",
    "    root=\".\",\n",
    "    download=True,\n",
    "    train=False,\n",
    "    transform = transforms.Compose([\n",
    "              transforms.ToTensor()\n",
    "          ])\n",
    ")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "256db298",
   "metadata": {},
   "outputs": [],
   "source": [
    "reqJson = json.loads('{\"inputs\":[{\"name\":\"Input3\",\"data\":[],\"datatype\":\"FP32\",\"shape\":[]}]}')\n",
    "url = \"http://0.0.0.0:9000/v2/models/model/infer\"\n",
    "dl = DataLoader(training_data, batch_size=1, shuffle=False)\n",
    "dlIter = iter(dl)\n",
    "\n",
    "def infer_mnist():\n",
    "    x, y = next(dlIter)\n",
    "    data = x.cpu().numpy()\n",
    "    reqJson[\"inputs\"][0][\"data\"] = data.flatten().tolist()\n",
    "    reqJson[\"inputs\"][0][\"shape\"] = [1, 1, 28, 28]\n",
    "    headers = {\"Content-Type\": \"application/json\", \"seldon-model\":\"mnist-onnx\"}\n",
    "    response_raw = requests.post(url, json=reqJson, headers=headers)\n",
    "    show_mnist(x)\n",
    "    probs = np.array(response_raw.json()[\"outputs\"][0][\"data\"])\n",
    "    print(probs.argmax(axis=0))\n",
    "    \n",
    "\n",
    "def show_mnist(X):\n",
    "    plt.imshow(X.reshape(28, 28))\n",
    "    plt.axis(\"off\")\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "9c5cd5af",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "apiVersion: mlops.seldon.io/v1alpha1\r\n",
      "kind: Model\r\n",
      "metadata:\r\n",
      "  name: mnist-onnx\r\n",
      "spec:\r\n",
      "  storageUri: \"gs://seldon-models/scv2/samples/triton_23-03/mnist-onnx\"\r\n",
      "  requirements:\r\n",
      "  - onnx\r\n"
     ]
    }
   ],
   "source": [
    "!cat ./models/mnist-onnx.yaml"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "8c698c9d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{}\r\n"
     ]
    }
   ],
   "source": [
    "!seldon model load -f ./models/mnist-onnx.yaml"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "91e05090",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{}\r\n"
     ]
    }
   ],
   "source": [
    "!seldon model status mnist-onnx -w ModelAvailable | jq -M ."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "bec480da",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAGFCAYAAAASI+9IAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/av/WaAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAIX0lEQVR4nO3cX6jfZQHH8e+ZZy43tczSSNNRoqSlV1YiwYJGhEgXOirIuzKqC+uiLgIhKIwkCcu6rAuxP0yDQCsQoSIylSKNTNBKQlqulqXm3Fzn1028ITI8z9eds+P2el3/Pnwf2PS95+ZZWiwWiwkApmnadKQPAMDGIQoARBQAiCgAEFEAIKIAQEQBgIgCAFle7Q93btq1lucAYI3dubL7BX/jpgBARAGAiAIAEQUAIgoARBQAiCgAEFEAIKIAQEQBgIgCABEFACIKAEQUAIgoABBRACCiAEBEAYCIAgARBQAiCgBEFACIKAAQUQAgogBARAGAiAIAEQUAIgoARBQAiCgAEFEAIKIAQEQBgIgCABEFACIKAEQUAIgoABBRACCiAEBEAYCIAgARBQAiCgBEFACIKAAQUQAgogBARAGAiAIAEQUAIgoARBQAiCgAEFEAIKIAQEQBgIgCABEFACIKAEQUAIgoABBRACCiAEBEAYCIAgARBQAiCgBEFACIKAAQUQAgogBARAGAiAIAEQUAIgoARBQAiCgAEFEAIKIAQEQBgIgCABEFACIKAEQUAIgoABBRACCiAEBEAYCIAgBZPtIHOBbs+9Alw5uzrnpk1rce2nv68Obggc3DmzO+Nb7Z+tjTw5tpmqaVXz04aweMc1MAIKIAQEQBgIgCABEFACIKAEQUAIgoABBRACCiAEBEAYCIAgARBQDildR18KlPfnN4c8W2J+Z97A3zZsN2jE8ePfTMrE/d+Jd3zNqxfu7de/bwZtsNL5/1reW7fjFrx+q4KQAQUQAgogBARAGAiAIAEQUAIgoARBQAiCgAEFEAIKIAQEQBgCwtFovFan64c9OutT7LUeufV751ePPXC+f1+pTfruqP87888cal4c3xF/59eHP9m747vJmmadp5wv7hzR3PnDi8uWzr08Ob9bR/cXB4c8+BbcObHS97bngzxzl3fHjW7tyr7zvMJzl23Lmy+wV/46YAQEQBgIgCABEFACIKAEQUAIgoABBRACCiAEBEAYCIAgARBQCyfKQPcCzYdus9MzZrcJD/4+R1+s5XXrNj1u5zl24f3pz840eGN9fvOGd4s56W968Mb7Y9sGd4c+pPbhvevPn4zcObrY+Ob1h7bgoARBQAiCgAEFEAIKIAQEQBgIgCABEFACIKAEQUAIgoABBRACAexGPdHPrz47N2224b3/1rzndu3TdjtbE9/sFLhjcXHD/+v4Uv/u284c32b/x+eDNN03Ro1orVclMAIKIAQEQBgIgCABEFACIKAEQUAIgoABBRACCiAEBEAYCIAgARBQDilVR4iVg++3XDm5s+fdPwZvPSccOb3Te+c3hz6p67hzesPTcFACIKAEQUAIgoABBRACCiAEBEAYCIAgARBQAiCgBEFACIKAAQD+LBS8RDnzhjeHPxlqXhzW8O7h/evPLBZ4Y3bExuCgBEFACIKAAQUQAgogBARAGAiAIAEQUAIgoARBQAiCgAEFEAIB7Eg3V24LKLZ+1+eeWXZqy2DC8+cs01w5sTfnbv8IaNyU0BgIgCABEFACIKAEQUAIgoABBRACCiAEBEAYCIAgARBQAiCgDEg3iwzv747nn/Fjtxafxxu/f/YefwZusP7x/eLIYXbFRuCgBEFACIKAAQUQAgogBARAGAiAIAEQUAIgoARBQAiCgAEFEAIB7Egxdh00knDW+uevtPZ33ryZVnhzd7r3v98GbLgfuGNxw93BQAiCgAEFEAIKIAQEQBgIgCABEFACIKAEQUAIgoABBRACCiAEBEAYB4JRVehIc/c8Hw5vZXfW3Wt97z8BXDmy3f9+IpY9wUAIgoABBRACCiAEBEAYCIAgARBQAiCgBEFACIKAAQUQAgogBAPIgH//GPD7xtePPAe788vPndoeeGN9M0TU9/4czhzZZpz6xvcexyUwAgogBARAGAiAIAEQUAIgoARBQAiCgAEFEAIKIAQEQBgIgCAPEgHkel5TNeO7z5+LXfGd5sWRr/T+h99181vJmmaXr1D+6btYMRbgoARBQAiCgAEFEAIKIAQEQBgIgCABEFACIKAEQUAIgoABBRACAexGPDW1oe/2t60e2PDW92nbhveHPLU6cNb06/dt6/xVZmrWCMmwIAEQUAIgoARBQAiCgAEFEAIKIAQEQBgIgCABEFACIKAEQUAIgH8dj4LjpvePLZ025eg4P8r69et2t484r7716Dk8Dh4aYAQEQBgIgCABEFACIKAEQUAIgoABBRACCiAEBEAYCIAgARBQAiCgDEK6msm+POP3fW7upvf+8wn+T5nf/1jw1vtt/88zU4CRw5bgoARBQAiCgAEFEAIKIAQEQBgIgCABEFACIKAEQUAIgoABBRACAexGPdPPTRU2btLt/65GE+yfM780cHx0eLxeE/CBxBbgoARBQAiCgAEFEAIKIAQEQBgIgCABEFACIKAEQUAIgoABBRACAexGOWZy9/y/DmrstvmPm1rTN3wCg3BQAiCgBEFACIKAAQUQAgogBARAGAiAIAEQUAIgoARBQAiCgAEA/iMcufLj1ueHPW8vo9bHfLU6cNbzY/eXB4sxhewMbmpgBARAGAiAIAEQUAIgoARBQAiCgAEFEAIKIAQEQBgIgCABEFACIKAMQrqWx4n993/vDm7ndtH94s9vx6eANHGzcFACIKAEQUAIgoABBRACCiAEBEAYCIAgARBQAiCgBEFACIKACQpcVisVjND3du2rXWZwFgDd25svsFf+OmAEBEAYCIAgARBQAiCgBEFACIKAAQUQAgogBARAGAiAIAEQUAsuoH8QA4+rkpABBRACCiAEBEAYCIAgARBQAiCgBEFACIKACQfwOvoLnvrbsiYAAAAABJRU5ErkJggg==",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "7\n"
     ]
    }
   ],
   "source": [
    "infer_mnist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "6a0cd431",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{}\r\n"
     ]
    }
   ],
   "source": [
    "!seldon model unload mnist-onnx"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8325a49e",
   "metadata": {},
   "source": [
    "### LightGBM Model\n",
    "\n",
    "The training code for this model can be found at `./scripts/models/income-lgb`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "47896ade",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "apiVersion: mlops.seldon.io/v1alpha1\r\n",
      "kind: Model\r\n",
      "metadata:\r\n",
      "  name: income-lgb\r\n",
      "spec:\r\n",
      "  storageUri: \"gs://seldon-models/scv2/samples/mlserver_1.3.5/income-lgb\"\r\n",
      "  requirements:\r\n",
      "  - lightgbm\r\n"
     ]
    }
   ],
   "source": [
    "!cat ./models/income-lgb.yaml"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "8ac81161",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{}\r\n"
     ]
    }
   ],
   "source": [
    "!seldon model load -f ./models/income-lgb.yaml"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "d65a7f91",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{}\r\n"
     ]
    }
   ],
   "source": [
    "!seldon model status income-lgb -w ModelAvailable | jq -M ."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "aa6403a1",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\r\n",
      "\t\"model_name\": \"income-lgb_1\",\r\n",
      "\t\"model_version\": \"1\",\r\n",
      "\t\"id\": \"4437a71e-9af1-4e3b-aa4b-cb95d2cd86b9\",\r\n",
      "\t\"parameters\": {},\r\n",
      "\t\"outputs\": [\r\n",
      "\t\t{\r\n",
      "\t\t\t\"name\": \"predict\",\r\n",
      "\t\t\t\"shape\": [\r\n",
      "\t\t\t\t1,\r\n",
      "\t\t\t\t1\r\n",
      "\t\t\t],\r\n",
      "\t\t\t\"datatype\": \"FP64\",\r\n",
      "\t\t\t\"data\": [\r\n",
      "\t\t\t\t0.06279460120044741\r\n",
      "\t\t\t]\r\n",
      "\t\t}\r\n",
      "\t]\r\n",
      "}\r\n"
     ]
    }
   ],
   "source": [
    "!seldon model infer income-lgb \\\n",
    "  '{ \"parameters\": {\"content_type\": \"pd\"}, \"inputs\": [{\"name\": \"Age\", \"shape\": [1, 1], \"datatype\": \"INT64\", \"data\": [47]},{\"name\": \"Workclass\", \"shape\": [1, 1], \"datatype\": \"INT64\", \"data\": [4]},{\"name\": \"Education\", \"shape\": [1, 1], \"datatype\": \"INT64\", \"data\": [1]},{\"name\": \"Marital Status\", \"shape\": [1, 1], \"datatype\": \"INT64\", \"data\": [1]},{\"name\": \"Occupation\", \"shape\": [1, 1], \"datatype\": \"INT64\", \"data\": [1]},{\"name\": \"Relationship\", \"shape\": [1, 1], \"datatype\": \"INT64\", \"data\": [3]},{\"name\": \"Race\", \"shape\": [1, 1], \"datatype\": \"INT64\", \"data\": [4]},{\"name\": \"Sex\", \"shape\": [1, 1], \"datatype\": \"INT64\", \"data\": [1]},{\"name\": \"Capital Gain\", \"shape\": [1, 1], \"datatype\": \"INT64\", \"data\": [0]},{\"name\": \"Capital Loss\", \"shape\": [1, 1], \"datatype\": \"INT64\", \"data\": [0]},{\"name\": \"Hours per week\", \"shape\": [1, 1], \"datatype\": \"INT64\", \"data\": [40]},{\"name\": \"Country\", \"shape\": [1, 1], \"datatype\": \"INT64\", \"data\": [9]}]}' "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "f1912a83",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{}\r\n"
     ]
    }
   ],
   "source": [
    "!seldon model unload income-lgb"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1baa5a6c",
   "metadata": {},
   "source": [
    "### MLFlow Wine Model\n",
    "\n",
    "The training code for this model can be found at `./scripts/models/wine-mlflow`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "5ffd0f32",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "apiVersion: mlops.seldon.io/v1alpha1\r\n",
      "kind: Model\r\n",
      "metadata:\r\n",
      "  name: wine\r\n",
      "spec:\r\n",
      "  storageUri: \"gs://seldon-models/scv2/samples/mlserver_1.3.5/wine-mlflow\"\r\n",
      "  requirements:\r\n",
      "  - mlflow\r\n"
     ]
    }
   ],
   "source": [
    "!cat ./models/wine-mlflow.yaml"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "8a5d38b6",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{}\r\n"
     ]
    }
   ],
   "source": [
    "!seldon model load -f ./models/wine-mlflow.yaml"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "2e749f0d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{}\r\n"
     ]
    }
   ],
   "source": [
    "!seldon model status wine -w ModelAvailable | jq -M ."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "c71e83de",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'model_name': 'wine_1', 'model_version': '1', 'id': '0d7e44f8-b46c-4438-b8af-a749e6aa6039', 'parameters': {}, 'outputs': [{'name': 'output-1', 'shape': [1, 1], 'datatype': 'FP64', 'data': [5.576883936610762]}]}\n"
     ]
    }
   ],
   "source": [
    "import requests\n",
    "url = \"http://0.0.0.0:9000/v2/models/model/infer\"\n",
    "inference_request = {\n",
    "    \"inputs\": [\n",
    "        {\n",
    "          \"name\": \"fixed acidity\",\n",
    "          \"shape\": [1],\n",
    "          \"datatype\": \"FP32\",\n",
    "          \"data\": [7.4],\n",
    "        },\n",
    "        {\n",
    "          \"name\": \"volatile acidity\",\n",
    "          \"shape\": [1],\n",
    "          \"datatype\": \"FP32\",\n",
    "          \"data\": [0.7000],\n",
    "        },\n",
    "        {\n",
    "          \"name\": \"citric acid\",\n",
    "          \"shape\": [1],\n",
    "          \"datatype\": \"FP32\",\n",
    "          \"data\": [0],\n",
    "        },\n",
    "        {\n",
    "          \"name\": \"residual sugar\",\n",
    "          \"shape\": [1],\n",
    "          \"datatype\": \"FP32\",\n",
    "          \"data\": [1.9],\n",
    "        },\n",
    "        {\n",
    "          \"name\": \"chlorides\",\n",
    "          \"shape\": [1],\n",
    "          \"datatype\": \"FP32\",\n",
    "          \"data\": [0.076],\n",
    "        },\n",
    "        {\n",
    "          \"name\": \"free sulfur dioxide\",\n",
    "          \"shape\": [1],\n",
    "          \"datatype\": \"FP32\",\n",
    "          \"data\": [11],\n",
    "        },\n",
    "        {\n",
    "          \"name\": \"total sulfur dioxide\",\n",
    "          \"shape\": [1],\n",
    "          \"datatype\": \"FP32\",\n",
    "          \"data\": [34],\n",
    "        },\n",
    "        {\n",
    "          \"name\": \"density\",\n",
    "          \"shape\": [1],\n",
    "          \"datatype\": \"FP32\",\n",
    "          \"data\": [0.9978],\n",
    "        },\n",
    "        {\n",
    "          \"name\": \"pH\",\n",
    "          \"shape\": [1],\n",
    "          \"datatype\": \"FP32\",\n",
    "          \"data\": [3.51],\n",
    "        },\n",
    "        {\n",
    "          \"name\": \"sulphates\",\n",
    "          \"shape\": [1],\n",
    "          \"datatype\": \"FP32\",\n",
    "          \"data\": [0.56],\n",
    "        },\n",
    "        {\n",
    "          \"name\": \"alcohol\",\n",
    "          \"shape\": [1],\n",
    "          \"datatype\": \"FP32\",\n",
    "          \"data\": [9.4],\n",
    "        },\n",
    "    ]\n",
    "}\n",
    "headers = {\"Content-Type\": \"application/json\", \"seldon-model\":\"wine\"}\n",
    "response_raw = requests.post(url, json=inference_request, headers=headers)\n",
    "print(response_raw.json())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "a2ba83ea",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{}\r\n"
     ]
    }
   ],
   "source": [
    "!seldon model unload wine"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d850d9dd",
   "metadata": {},
   "source": [
    "## Pytorch MNIST Model\n",
    "\n",
    "This example model is downloaded and trained in `./scripts/models/Makefile` target `mnist-pytorch`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "57d19f06",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import json\n",
    "import requests\n",
    "from torchvision.datasets import MNIST\n",
    "from torchvision.transforms import ToTensor\n",
    "from torchvision import transforms\n",
    "from torch.utils.data import DataLoader\n",
    "training_data = MNIST(\n",
    "    root=\".\",\n",
    "    download=True,\n",
    "    train=False,\n",
    "    transform = transforms.Compose([\n",
    "              transforms.ToTensor()\n",
    "          ])\n",
    ")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "5b27e8b5",
   "metadata": {},
   "outputs": [],
   "source": [
    "reqJson = json.loads('{\"inputs\":[{\"name\":\"x__0\",\"data\":[],\"datatype\":\"FP32\",\"shape\":[]}]}')\n",
    "url = \"http://0.0.0.0:9000/v2/models/model/infer\"\n",
    "dl = DataLoader(training_data, batch_size=1, shuffle=False)\n",
    "dlIter = iter(dl)\n",
    "\n",
    "def infer_mnist():\n",
    "    x, y = next(dlIter)\n",
    "    data = x.cpu().numpy()\n",
    "    reqJson[\"inputs\"][0][\"data\"] = data.flatten().tolist()\n",
    "    reqJson[\"inputs\"][0][\"shape\"] = [1, 1, 28, 28]\n",
    "    headers = {\"Content-Type\": \"application/json\", \"seldon-model\":\"mnist-pytorch\"}\n",
    "    response_raw = requests.post(url, json=reqJson, headers=headers)\n",
    "    show_mnist(x)\n",
    "    probs = np.array(response_raw.json()[\"outputs\"][0][\"data\"])\n",
    "    print(probs.argmax(axis=0))\n",
    "    \n",
    "\n",
    "def show_mnist(X):\n",
    "    plt.imshow(X.reshape(28, 28))\n",
    "    plt.axis(\"off\")\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "a5085df5",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "apiVersion: mlops.seldon.io/v1alpha1\r\n",
      "kind: Model\r\n",
      "metadata:\r\n",
      "  name: mnist-pytorch\r\n",
      "spec:\r\n",
      "  storageUri: \"gs://seldon-models/scv2/samples/triton_23-03/mnist-pytorch\"\r\n",
      "  requirements:\r\n",
      "  - pytorch\r\n"
     ]
    }
   ],
   "source": [
    "!cat ./models/mnist-pytorch.yaml"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "ada997dd",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{}\r\n"
     ]
    }
   ],
   "source": [
    "!seldon model load -f ./models/mnist-pytorch.yaml"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "85e443ac",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{}\r\n"
     ]
    }
   ],
   "source": [
    "!seldon model status mnist-pytorch -w ModelAvailable | jq -M ."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "a8f1ba5c",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAGFCAYAAAASI+9IAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/av/WaAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAIX0lEQVR4nO3cX6jfZQHH8e+ZZy43tczSSNNRoqSlV1YiwYJGhEgXOirIuzKqC+uiLgIhKIwkCcu6rAuxP0yDQCsQoSIylSKNTNBKQlqulqXm3Fzn1028ITI8z9eds+P2el3/Pnwf2PS95+ZZWiwWiwkApmnadKQPAMDGIQoARBQAiCgAEFEAIKIAQEQBgIgCAFle7Q93btq1lucAYI3dubL7BX/jpgBARAGAiAIAEQUAIgoARBQAiCgAEFEAIKIAQEQBgIgCABEFACIKAEQUAIgoABBRACCiAEBEAYCIAgARBQAiCgBEFACIKAAQUQAgogBARAGAiAIAEQUAIgoARBQAiCgAEFEAIKIAQEQBgIgCABEFACIKAEQUAIgoABBRACCiAEBEAYCIAgARBQAiCgBEFACIKAAQUQAgogBARAGAiAIAEQUAIgoARBQAiCgAEFEAIKIAQEQBgIgCABEFACIKAEQUAIgoABBRACCiAEBEAYCIAgARBQAiCgBEFACIKAAQUQAgogBARAGAiAIAEQUAIgoARBQAiCgAEFEAIKIAQEQBgIgCABEFACIKAEQUAIgoABBRACCiAEBEAYCIAgBZPtIHOBbs+9Alw5uzrnpk1rce2nv68Obggc3DmzO+Nb7Z+tjTw5tpmqaVXz04aweMc1MAIKIAQEQBgIgCABEFACIKAEQUAIgoABBRACCiAEBEAYCIAgARBQDildR18KlPfnN4c8W2J+Z97A3zZsN2jE8ePfTMrE/d+Jd3zNqxfu7de/bwZtsNL5/1reW7fjFrx+q4KQAQUQAgogBARAGAiAIAEQUAIgoARBQAiCgAEFEAIKIAQEQBgCwtFovFan64c9OutT7LUeufV751ePPXC+f1+pTfruqP87888cal4c3xF/59eHP9m747vJmmadp5wv7hzR3PnDi8uWzr08Ob9bR/cXB4c8+BbcObHS97bngzxzl3fHjW7tyr7zvMJzl23Lmy+wV/46YAQEQBgIgCABEFACIKAEQUAIgoABBRACCiAEBEAYCIAgARBQCyfKQPcCzYdus9MzZrcJD/4+R1+s5XXrNj1u5zl24f3pz840eGN9fvOGd4s56W968Mb7Y9sGd4c+pPbhvevPn4zcObrY+Ob1h7bgoARBQAiCgAEFEAIKIAQEQBgIgCABEFACIKAEQUAIgoABBRACAexGPdHPrz47N2224b3/1rzndu3TdjtbE9/sFLhjcXHD/+v4Uv/u284c32b/x+eDNN03Ro1orVclMAIKIAQEQBgIgCABEFACIKAEQUAIgoABBRACCiAEBEAYCIAgARBQDilVR4iVg++3XDm5s+fdPwZvPSccOb3Te+c3hz6p67hzesPTcFACIKAEQUAIgoABBRACCiAEBEAYCIAgARBQAiCgBEFACIKAAQD+LBS8RDnzhjeHPxlqXhzW8O7h/evPLBZ4Y3bExuCgBEFACIKAAQUQAgogBARAGAiAIAEQUAIgoARBQAiCgAEFEAIB7Eg3V24LKLZ+1+eeWXZqy2DC8+cs01w5sTfnbv8IaNyU0BgIgCABEFACIKAEQUAIgoABBRACCiAEBEAYCIAgARBQAiCgDEg3iwzv747nn/Fjtxafxxu/f/YefwZusP7x/eLIYXbFRuCgBEFACIKAAQUQAgogBARAGAiAIAEQUAIgoARBQAiCgAEFEAIB7Egxdh00knDW+uevtPZ33ryZVnhzd7r3v98GbLgfuGNxw93BQAiCgAEFEAIKIAQEQBgIgCABEFACIKAEQUAIgoABBRACCiAEBEAYB4JRVehIc/c8Hw5vZXfW3Wt97z8BXDmy3f9+IpY9wUAIgoABBRACCiAEBEAYCIAgARBQAiCgBEFACIKAAQUQAgogBAPIgH//GPD7xtePPAe788vPndoeeGN9M0TU9/4czhzZZpz6xvcexyUwAgogBARAGAiAIAEQUAIgoARBQAiCgAEFEAIKIAQEQBgIgCAPEgHkel5TNeO7z5+LXfGd5sWRr/T+h99181vJmmaXr1D+6btYMRbgoARBQAiCgAEFEAIKIAQEQBgIgCABEFACIKAEQUAIgoABBRACAexGPDW1oe/2t60e2PDW92nbhveHPLU6cNb06/dt6/xVZmrWCMmwIAEQUAIgoARBQAiCgAEFEAIKIAQEQBgIgCABEFACIKAEQUAIgH8dj4LjpvePLZ025eg4P8r69et2t484r7716Dk8Dh4aYAQEQBgIgCABEFACIKAEQUAIgoABBRACCiAEBEAYCIAgARBQAiCgDEK6msm+POP3fW7upvf+8wn+T5nf/1jw1vtt/88zU4CRw5bgoARBQAiCgAEFEAIKIAQEQBgIgCABEFACIKAEQUAIgoABBRACAexGPdPPTRU2btLt/65GE+yfM780cHx0eLxeE/CBxBbgoARBQAiCgAEFEAIKIAQEQBgIgCABEFACIKAEQUAIgoABBRACAexGOWZy9/y/DmrstvmPm1rTN3wCg3BQAiCgBEFACIKAAQUQAgogBARAGAiAIAEQUAIgoARBQAiCgAEA/iMcufLj1ueHPW8vo9bHfLU6cNbzY/eXB4sxhewMbmpgBARAGAiAIAEQUAIgoARBQAiCgAEFEAIKIAQEQBgIgCABEFACIKAMQrqWx4n993/vDm7ndtH94s9vx6eANHGzcFACIKAEQUAIgoABBRACCiAEBEAYCIAgARBQAiCgBEFACIKACQpcVisVjND3du2rXWZwFgDd25svsFf+OmAEBEAYCIAgARBQAiCgBEFACIKAAQUQAgogBARAGAiAIAEQUAsuoH8QA4+rkpABBRACCiAEBEAYCIAgARBQAiCgBEFACIKACQfwOvoLnvrbsiYAAAAABJRU5ErkJggg==",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "7\n"
     ]
    }
   ],
   "source": [
    "infer_mnist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "b972b1ef",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{}\r\n"
     ]
    }
   ],
   "source": [
    "!seldon model unload mnist-pytorch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "94cd2aba",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
